{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "!pip install --upgrade wandb\n",
    "!wandb login 3da7a23df9fd940d985adf808de2b09ceb85f15b\n",
    "\n",
    "import wandb\n",
    "wandb.init(project=\"global-wheat-detection\", name='FasterRCNN with ResNet101 backbone: fold1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "%%capture\n",
    "!pip install cython\n",
    "!pip install 'git+https://github.com/cocodataset/cocoapi.git#subdirectory=PythonAPI'\n",
    "!cp /kaggle/input/rcnnutilswithwandb/engine.py .\n",
    "!cp /kaggle/input/rcnnutilswithwandb/utils.py .\n",
    "!cp /kaggle/input/rcnnutilswithwandb/coco_eval.py .\n",
    "!cp /kaggle/input/rcnnutilswithwandb/coco_utils.py .\n",
    "!cp /kaggle/input/rcnnutilswithwandb/transforms.py ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import ast\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "import cv2\n",
    "\n",
    "import torch\n",
    "from PIL import Image\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "import albumentations\n",
    "from albumentations.pytorch.transforms import ToTensorV2\n",
    "\n",
    "from torch import nn\n",
    "import torchvision\n",
    "import torch.utils.data as data_utils\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "from torchvision.models.detection.faster_rcnn import FastRCNNPredictor\n",
    "from torchvision.models.detection import FasterRCNN\n",
    "from torchvision.models.detection.rpn import AnchorGenerator\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "import matplotlib.patches as patches\n",
    "\n",
    "import utils\n",
    "from engine import train_one_epoch, evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Constants\n",
    "TEST_DIR = '/kaggle/input/global-wheat-detection/test'\n",
    "BASE_DIR = '/kaggle/input/gwdaugmented/train'\n",
    "BATCH_SIZE = 2\n",
    "DEVICE = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
    "# our dataset has two classes only - background and wheat heads\n",
    "N_CLASSES = 2\n",
    "N_EPOCHS = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>x_min</th>\n",
       "      <th>y_min</th>\n",
       "      <th>x_max</th>\n",
       "      <th>y_max</th>\n",
       "      <th>width</th>\n",
       "      <th>height</th>\n",
       "      <th>area</th>\n",
       "      <th>source</th>\n",
       "      <th>kfold</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b6ab77fd7</td>\n",
       "      <td>834.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>890.0</td>\n",
       "      <td>258.0</td>\n",
       "      <td>56.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>usask_1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>b6ab77fd7</td>\n",
       "      <td>226.0</td>\n",
       "      <td>548.0</td>\n",
       "      <td>356.0</td>\n",
       "      <td>606.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>7540.0</td>\n",
       "      <td>usask_1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b6ab77fd7</td>\n",
       "      <td>377.0</td>\n",
       "      <td>504.0</td>\n",
       "      <td>451.0</td>\n",
       "      <td>664.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>160.0</td>\n",
       "      <td>11840.0</td>\n",
       "      <td>usask_1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b6ab77fd7</td>\n",
       "      <td>834.0</td>\n",
       "      <td>95.0</td>\n",
       "      <td>943.0</td>\n",
       "      <td>202.0</td>\n",
       "      <td>109.0</td>\n",
       "      <td>107.0</td>\n",
       "      <td>11663.0</td>\n",
       "      <td>usask_1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>b6ab77fd7</td>\n",
       "      <td>26.0</td>\n",
       "      <td>144.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>261.0</td>\n",
       "      <td>124.0</td>\n",
       "      <td>117.0</td>\n",
       "      <td>14508.0</td>\n",
       "      <td>usask_1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295533</th>\n",
       "      <td>5e0747034_aug</td>\n",
       "      <td>876.0</td>\n",
       "      <td>619.0</td>\n",
       "      <td>960.0</td>\n",
       "      <td>714.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>95.0</td>\n",
       "      <td>7980.0</td>\n",
       "      <td>arvalis_2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295534</th>\n",
       "      <td>5e0747034_aug</td>\n",
       "      <td>625.0</td>\n",
       "      <td>549.0</td>\n",
       "      <td>732.0</td>\n",
       "      <td>631.0</td>\n",
       "      <td>107.0</td>\n",
       "      <td>82.0</td>\n",
       "      <td>8774.0</td>\n",
       "      <td>arvalis_2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295535</th>\n",
       "      <td>5e0747034_aug</td>\n",
       "      <td>749.0</td>\n",
       "      <td>228.0</td>\n",
       "      <td>890.0</td>\n",
       "      <td>299.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>10011.0</td>\n",
       "      <td>arvalis_2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295536</th>\n",
       "      <td>5e0747034_aug</td>\n",
       "      <td>410.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>594.0</td>\n",
       "      <td>92.0</td>\n",
       "      <td>184.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>14536.0</td>\n",
       "      <td>arvalis_2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295537</th>\n",
       "      <td>5e0747034_aug</td>\n",
       "      <td>55.0</td>\n",
       "      <td>740.0</td>\n",
       "      <td>149.0</td>\n",
       "      <td>801.0</td>\n",
       "      <td>94.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>5734.0</td>\n",
       "      <td>arvalis_2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>295538 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             image_id  x_min  y_min  x_max  y_max  width  height     area  \\\n",
       "0           b6ab77fd7  834.0  222.0  890.0  258.0   56.0    36.0   2016.0   \n",
       "1           b6ab77fd7  226.0  548.0  356.0  606.0  130.0    58.0   7540.0   \n",
       "2           b6ab77fd7  377.0  504.0  451.0  664.0   74.0   160.0  11840.0   \n",
       "3           b6ab77fd7  834.0   95.0  943.0  202.0  109.0   107.0  11663.0   \n",
       "4           b6ab77fd7   26.0  144.0  150.0  261.0  124.0   117.0  14508.0   \n",
       "...               ...    ...    ...    ...    ...    ...     ...      ...   \n",
       "295533  5e0747034_aug  876.0  619.0  960.0  714.0   84.0    95.0   7980.0   \n",
       "295534  5e0747034_aug  625.0  549.0  732.0  631.0  107.0    82.0   8774.0   \n",
       "295535  5e0747034_aug  749.0  228.0  890.0  299.0  141.0    71.0  10011.0   \n",
       "295536  5e0747034_aug  410.0   13.0  594.0   92.0  184.0    79.0  14536.0   \n",
       "295537  5e0747034_aug   55.0  740.0  149.0  801.0   94.0    61.0   5734.0   \n",
       "\n",
       "           source  kfold  \n",
       "0         usask_1      2  \n",
       "1         usask_1      2  \n",
       "2         usask_1      2  \n",
       "3         usask_1      2  \n",
       "4         usask_1      2  \n",
       "...           ...    ...  \n",
       "295533  arvalis_2      1  \n",
       "295534  arvalis_2      1  \n",
       "295535  arvalis_2      1  \n",
       "295536  arvalis_2      1  \n",
       "295537  arvalis_2      1  \n",
       "\n",
       "[295538 rows x 10 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = pd.read_csv(os.path.join('/kaggle/input/gwdaugmented/', 'train.csv'))\n",
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>x_min</th>\n",
       "      <th>y_min</th>\n",
       "      <th>x_max</th>\n",
       "      <th>y_max</th>\n",
       "      <th>width</th>\n",
       "      <th>height</th>\n",
       "      <th>area</th>\n",
       "      <th>source</th>\n",
       "      <th>kfold</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b6ab77fd7</td>\n",
       "      <td>834.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>890.0</td>\n",
       "      <td>258.0</td>\n",
       "      <td>56.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>usask_1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>b6ab77fd7</td>\n",
       "      <td>226.0</td>\n",
       "      <td>548.0</td>\n",
       "      <td>356.0</td>\n",
       "      <td>606.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>7540.0</td>\n",
       "      <td>usask_1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b6ab77fd7</td>\n",
       "      <td>377.0</td>\n",
       "      <td>504.0</td>\n",
       "      <td>451.0</td>\n",
       "      <td>664.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>160.0</td>\n",
       "      <td>11840.0</td>\n",
       "      <td>usask_1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b6ab77fd7</td>\n",
       "      <td>834.0</td>\n",
       "      <td>95.0</td>\n",
       "      <td>943.0</td>\n",
       "      <td>202.0</td>\n",
       "      <td>109.0</td>\n",
       "      <td>107.0</td>\n",
       "      <td>11663.0</td>\n",
       "      <td>usask_1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>b6ab77fd7</td>\n",
       "      <td>26.0</td>\n",
       "      <td>144.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>261.0</td>\n",
       "      <td>124.0</td>\n",
       "      <td>117.0</td>\n",
       "      <td>14508.0</td>\n",
       "      <td>usask_1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295533</th>\n",
       "      <td>5e0747034_aug</td>\n",
       "      <td>876.0</td>\n",
       "      <td>619.0</td>\n",
       "      <td>960.0</td>\n",
       "      <td>714.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>95.0</td>\n",
       "      <td>7980.0</td>\n",
       "      <td>arvalis_2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295534</th>\n",
       "      <td>5e0747034_aug</td>\n",
       "      <td>625.0</td>\n",
       "      <td>549.0</td>\n",
       "      <td>732.0</td>\n",
       "      <td>631.0</td>\n",
       "      <td>107.0</td>\n",
       "      <td>82.0</td>\n",
       "      <td>8774.0</td>\n",
       "      <td>arvalis_2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295535</th>\n",
       "      <td>5e0747034_aug</td>\n",
       "      <td>749.0</td>\n",
       "      <td>228.0</td>\n",
       "      <td>890.0</td>\n",
       "      <td>299.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>10011.0</td>\n",
       "      <td>arvalis_2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295536</th>\n",
       "      <td>5e0747034_aug</td>\n",
       "      <td>410.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>594.0</td>\n",
       "      <td>92.0</td>\n",
       "      <td>184.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>14536.0</td>\n",
       "      <td>arvalis_2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295537</th>\n",
       "      <td>5e0747034_aug</td>\n",
       "      <td>55.0</td>\n",
       "      <td>740.0</td>\n",
       "      <td>149.0</td>\n",
       "      <td>801.0</td>\n",
       "      <td>94.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>5734.0</td>\n",
       "      <td>arvalis_2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>295538 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             image_id  x_min  y_min  x_max  y_max  width  height     area  \\\n",
       "0           b6ab77fd7  834.0  222.0  890.0  258.0   56.0    36.0   2016.0   \n",
       "1           b6ab77fd7  226.0  548.0  356.0  606.0  130.0    58.0   7540.0   \n",
       "2           b6ab77fd7  377.0  504.0  451.0  664.0   74.0   160.0  11840.0   \n",
       "3           b6ab77fd7  834.0   95.0  943.0  202.0  109.0   107.0  11663.0   \n",
       "4           b6ab77fd7   26.0  144.0  150.0  261.0  124.0   117.0  14508.0   \n",
       "...               ...    ...    ...    ...    ...    ...     ...      ...   \n",
       "295533  5e0747034_aug  876.0  619.0  960.0  714.0   84.0    95.0   7980.0   \n",
       "295534  5e0747034_aug  625.0  549.0  732.0  631.0  107.0    82.0   8774.0   \n",
       "295535  5e0747034_aug  749.0  228.0  890.0  299.0  141.0    71.0  10011.0   \n",
       "295536  5e0747034_aug  410.0   13.0  594.0   92.0  184.0    79.0  14536.0   \n",
       "295537  5e0747034_aug   55.0  740.0  149.0  801.0   94.0    61.0   5734.0   \n",
       "\n",
       "           source  kfold  \n",
       "0         usask_1      2  \n",
       "1         usask_1      2  \n",
       "2         usask_1      2  \n",
       "3         usask_1      2  \n",
       "4         usask_1      2  \n",
       "...           ...    ...  \n",
       "295533  arvalis_2      1  \n",
       "295534  arvalis_2      1  \n",
       "295535  arvalis_2      1  \n",
       "295536  arvalis_2      1  \n",
       "295537  arvalis_2      1  \n",
       "\n",
       "[295538 rows x 10 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class WheatDataset(Dataset):\n",
    "    \n",
    "    def __init__(self, df, folds, transforms=None):\n",
    "        self.df = df[df.kfold.isin(folds)].reset_index(drop=True)\n",
    "        self.image_ids = self.df['image_id'].unique()\n",
    "        self.transforms = transforms\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.image_ids)\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        image_id = self.image_ids[index]\n",
    "        image = cv2.imread(os.path.join(BASE_DIR, 'train', f'{image_id}.jpg'), cv2.IMREAD_COLOR)\n",
    "        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n",
    "        image /= 255.0\n",
    "\n",
    "        # Convert from NHWC to NCHW as pytorch expects images in NCHW format\n",
    "        image = np.transpose(image, (2, 0, 1))\n",
    "        image = torch.from_numpy(image)\n",
    "        \n",
    "        # Get bbox coordinates for each wheat head(s)\n",
    "        bboxes_df = self.df[self.df['image_id'] == image_id]\n",
    "        boxes, areas = [], []\n",
    "        n_objects = len(bboxes_df)  # Number of wheat heads in the given image\n",
    "\n",
    "        for i in range(n_objects):\n",
    "            x_min = bboxes_df.iloc[i]['x_min']\n",
    "            x_max = bboxes_df.iloc[i]['x_max']\n",
    "            y_min = bboxes_df.iloc[i]['y_min']\n",
    "            y_max = bboxes_df.iloc[i]['y_max']\n",
    "\n",
    "            boxes.append([x_min, y_min, x_max, y_max])\n",
    "            areas.append(bboxes_df.iloc[i]['area'])\n",
    "\n",
    "        boxes = torch.as_tensor(boxes, dtype=torch.int64)\n",
    "        \n",
    "        # Get the labels. We have only one class (wheat head)\n",
    "        labels = torch.ones((n_objects, ), dtype=torch.int64)\n",
    "        \n",
    "        areas = torch.as_tensor(areas)\n",
    "        \n",
    "        # suppose all instances are not crowd\n",
    "        iscrowd = torch.zeros((n_objects, ), dtype=torch.int64)\n",
    "        \n",
    "        target = {\n",
    "            'boxes': boxes,\n",
    "            'labels': labels,\n",
    "            'image_id': torch.tensor([index]),\n",
    "            'area': areas,\n",
    "            'iscrowd': iscrowd\n",
    "        }\n",
    "        \n",
    "        if self.transforms:\n",
    "            result_aug = self.transforms(image=image, bboxes=boxes, labels=labels)\n",
    "            image = result_aug['image'].float()\n",
    "            \n",
    "            target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*result_aug['bboxes'])))).permute(1, 0)\n",
    "\n",
    "        return image, target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model(pre_trained=True):\n",
    "    \n",
    "    # Reference: https://stackoverflow.com/questions/58362892/resnet-18-as-backbone-in-faster-r-cnn\n",
    "    resnet_net = torchvision.models.resnet101(pretrained=True) \n",
    "    modules = list(resnet_net.children())[:-2]\n",
    "\n",
    "    backbone = nn.Sequential(*modules)\n",
    "    backbone.out_channels = 2048\n",
    "\n",
    "    # let's make the RPN generate 5 x 3 anchors per spatial\n",
    "    # location, with 5 different sizes and 3 different aspect\n",
    "    # ratios. We have a Tuple[Tuple[int]] because each feature\n",
    "    # map could potentially have different sizes and\n",
    "    # aspect ratios\n",
    "    anchor_generator = AnchorGenerator(sizes=((32, 64, 128, 256, 512),),\n",
    "                                       aspect_ratios=((0.5, 1.0, 2.0),))\n",
    "\n",
    "    # put the pieces together inside a FasterRCNN model\n",
    "    model = FasterRCNN(backbone,\n",
    "                       num_classes=N_CLASSES,\n",
    "                       rpn_anchor_generator=anchor_generator)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "# get the model using our helper function\n",
    "model = get_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:2854: UserWarning: The default behavior for interpolate/upsample with float scale_factor will change in 1.6.0 to align with other frameworks/libraries, and use scale_factor directly, instead of relying on the computed output size. If you wish to keep the old behavior, please set recompute_scale_factor=True. See the documentation of nn.Upsample for details. \n",
      "  warnings.warn(\"The default behavior for interpolate/upsample with float scale_factor will change \"\n",
      "/opt/conda/conda-bld/pytorch_1587428398394/work/torch/csrc/utils/python_arg_parser.cpp:756: UserWarning: This overload of nonzero is deprecated:\n",
      "\tnonzero(Tensor input, *, Tensor out)\n",
      "Consider using one of the following signatures instead:\n",
      "\tnonzero(Tensor input, *, bool as_tuple)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [0]  [   0/2699]  eta: 2:42:08  lr: 0.000010  loss: 1.7506 (1.7506)  loss_classifier: 0.6703 (0.6703)  loss_box_reg: 0.1507 (0.1507)  loss_objectness: 0.6973 (0.6973)  loss_rpn_box_reg: 0.2324 (0.2324)  time: 3.6045  data: 1.0496  max mem: 5070\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:2854: UserWarning: The default behavior for interpolate/upsample with float scale_factor will change in 1.6.0 to align with other frameworks/libraries, and use scale_factor directly, instead of relying on the computed output size. If you wish to keep the old behavior, please set recompute_scale_factor=True. See the documentation of nn.Upsample for details. \n",
      "  warnings.warn(\"The default behavior for interpolate/upsample with float scale_factor will change \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [0]  [ 100/2699]  eta: 0:23:44  lr: 0.000509  loss: 1.2325 (1.3954)  loss_classifier: 0.4274 (0.4789)  loss_box_reg: 0.2686 (0.2054)  loss_objectness: 0.3429 (0.4953)  loss_rpn_box_reg: 0.2105 (0.2157)  time: 0.5157  data: 0.0105  max mem: 6482\n",
      "Epoch: [0]  [ 200/2699]  eta: 0:22:12  lr: 0.001009  loss: 1.1705 (1.2868)  loss_classifier: 0.3662 (0.4343)  loss_box_reg: 0.3261 (0.2499)  loss_objectness: 0.2666 (0.4028)  loss_rpn_box_reg: 0.1703 (0.1997)  time: 0.5142  data: 0.0103  max mem: 6482\n",
      "Epoch: [0]  [ 300/2699]  eta: 0:21:06  lr: 0.001508  loss: 1.1938 (1.2395)  loss_classifier: 0.3949 (0.4165)  loss_box_reg: 0.4145 (0.2908)  loss_objectness: 0.2124 (0.3474)  loss_rpn_box_reg: 0.1432 (0.1848)  time: 0.5251  data: 0.0122  max mem: 6482\n",
      "Epoch: [0]  [ 400/2699]  eta: 0:20:04  lr: 0.002008  loss: 1.0916 (1.1980)  loss_classifier: 0.3729 (0.4030)  loss_box_reg: 0.3944 (0.3133)  loss_objectness: 0.1866 (0.3076)  loss_rpn_box_reg: 0.1581 (0.1740)  time: 0.5153  data: 0.0110  max mem: 6482\n",
      "Epoch: [0]  [ 500/2699]  eta: 0:19:07  lr: 0.002507  loss: 0.9700 (1.1612)  loss_classifier: 0.3227 (0.3915)  loss_box_reg: 0.3766 (0.3253)  loss_objectness: 0.1312 (0.2776)  loss_rpn_box_reg: 0.1130 (0.1668)  time: 0.5133  data: 0.0110  max mem: 6482\n",
      "Epoch: [0]  [ 600/2699]  eta: 0:18:13  lr: 0.003007  loss: 0.9199 (1.1169)  loss_classifier: 0.3183 (0.3776)  loss_box_reg: 0.3526 (0.3292)  loss_objectness: 0.1144 (0.2511)  loss_rpn_box_reg: 0.1252 (0.1590)  time: 0.5135  data: 0.0109  max mem: 6482\n",
      "Epoch: [0]  [ 700/2699]  eta: 0:17:19  lr: 0.003506  loss: 0.8918 (1.0859)  loss_classifier: 0.3175 (0.3679)  loss_box_reg: 0.3404 (0.3311)  loss_objectness: 0.1161 (0.2326)  loss_rpn_box_reg: 0.1037 (0.1543)  time: 0.5122  data: 0.0100  max mem: 6482\n",
      "Epoch: [0]  [ 800/2699]  eta: 0:16:26  lr: 0.004006  loss: 0.8054 (1.0532)  loss_classifier: 0.2739 (0.3581)  loss_box_reg: 0.3293 (0.3303)  loss_objectness: 0.0948 (0.2160)  loss_rpn_box_reg: 0.1109 (0.1488)  time: 0.5130  data: 0.0104  max mem: 6482\n",
      "Epoch: [0]  [ 900/2699]  eta: 0:15:33  lr: 0.004505  loss: 0.7691 (1.0243)  loss_classifier: 0.2851 (0.3498)  loss_box_reg: 0.2997 (0.3282)  loss_objectness: 0.0783 (0.2022)  loss_rpn_box_reg: 0.1042 (0.1440)  time: 0.5142  data: 0.0106  max mem: 6482\n",
      "Epoch: [0]  [1000/2699]  eta: 0:14:41  lr: 0.005000  loss: 0.7387 (0.9949)  loss_classifier: 0.2769 (0.3415)  loss_box_reg: 0.3033 (0.3246)  loss_objectness: 0.0796 (0.1904)  loss_rpn_box_reg: 0.0831 (0.1385)  time: 0.5238  data: 0.0111  max mem: 6482\n",
      "Epoch: [0]  [1100/2699]  eta: 0:13:48  lr: 0.005000  loss: 0.7564 (0.9726)  loss_classifier: 0.2681 (0.3350)  loss_box_reg: 0.2942 (0.3218)  loss_objectness: 0.0713 (0.1809)  loss_rpn_box_reg: 0.0872 (0.1349)  time: 0.5239  data: 0.0133  max mem: 6482\n",
      "Epoch: [0]  [1200/2699]  eta: 0:12:56  lr: 0.005000  loss: 0.7162 (0.9521)  loss_classifier: 0.2652 (0.3292)  loss_box_reg: 0.2870 (0.3189)  loss_objectness: 0.0695 (0.1726)  loss_rpn_box_reg: 0.0841 (0.1314)  time: 0.5158  data: 0.0111  max mem: 6482\n",
      "Epoch: [0]  [1300/2699]  eta: 0:12:04  lr: 0.005000  loss: 0.7478 (0.9342)  loss_classifier: 0.2679 (0.3241)  loss_box_reg: 0.2762 (0.3162)  loss_objectness: 0.0798 (0.1653)  loss_rpn_box_reg: 0.0998 (0.1286)  time: 0.5138  data: 0.0105  max mem: 6482\n",
      "Epoch: [0]  [1400/2699]  eta: 0:11:13  lr: 0.005000  loss: 0.6117 (0.9170)  loss_classifier: 0.2358 (0.3192)  loss_box_reg: 0.2493 (0.3127)  loss_objectness: 0.0619 (0.1590)  loss_rpn_box_reg: 0.0709 (0.1261)  time: 0.5146  data: 0.0110  max mem: 6482\n",
      "Epoch: [0]  [1500/2699]  eta: 0:10:21  lr: 0.005000  loss: 0.6948 (0.9001)  loss_classifier: 0.2539 (0.3146)  loss_box_reg: 0.2723 (0.3092)  loss_objectness: 0.0673 (0.1531)  loss_rpn_box_reg: 0.0883 (0.1232)  time: 0.5156  data: 0.0113  max mem: 6482\n",
      "Epoch: [0]  [1600/2699]  eta: 0:09:29  lr: 0.005000  loss: 0.7101 (0.8862)  loss_classifier: 0.2708 (0.3109)  loss_box_reg: 0.2717 (0.3065)  loss_objectness: 0.0633 (0.1478)  loss_rpn_box_reg: 0.1075 (0.1210)  time: 0.5148  data: 0.0108  max mem: 6482\n",
      "Epoch: [0]  [1700/2699]  eta: 0:08:38  lr: 0.005000  loss: 0.6428 (0.8732)  loss_classifier: 0.2481 (0.3075)  loss_box_reg: 0.2396 (0.3035)  loss_objectness: 0.0649 (0.1433)  loss_rpn_box_reg: 0.0918 (0.1189)  time: 0.5301  data: 0.0125  max mem: 6482\n",
      "Epoch: [0]  [1800/2699]  eta: 0:07:46  lr: 0.005000  loss: 0.6553 (0.8605)  loss_classifier: 0.2408 (0.3042)  loss_box_reg: 0.2481 (0.3008)  loss_objectness: 0.0641 (0.1390)  loss_rpn_box_reg: 0.0778 (0.1166)  time: 0.5224  data: 0.0129  max mem: 6482\n",
      "Epoch: [0]  [1900/2699]  eta: 0:06:54  lr: 0.005000  loss: 0.6314 (0.8492)  loss_classifier: 0.2418 (0.3008)  loss_box_reg: 0.2405 (0.2980)  loss_objectness: 0.0594 (0.1353)  loss_rpn_box_reg: 0.0786 (0.1151)  time: 0.5132  data: 0.0105  max mem: 6482\n",
      "Epoch: [0]  [2000/2699]  eta: 0:06:02  lr: 0.005000  loss: 0.6249 (0.8398)  loss_classifier: 0.2484 (0.2984)  loss_box_reg: 0.2524 (0.2958)  loss_objectness: 0.0600 (0.1319)  loss_rpn_box_reg: 0.0725 (0.1136)  time: 0.5193  data: 0.0130  max mem: 6482\n",
      "Epoch: [0]  [2100/2699]  eta: 0:05:10  lr: 0.005000  loss: 0.6007 (0.8293)  loss_classifier: 0.2233 (0.2954)  loss_box_reg: 0.2323 (0.2931)  loss_objectness: 0.0546 (0.1287)  loss_rpn_box_reg: 0.0659 (0.1120)  time: 0.5154  data: 0.0110  max mem: 6482\n",
      "Epoch: [0]  [2200/2699]  eta: 0:04:18  lr: 0.005000  loss: 0.6142 (0.8192)  loss_classifier: 0.2220 (0.2925)  loss_box_reg: 0.2260 (0.2907)  loss_objectness: 0.0530 (0.1255)  loss_rpn_box_reg: 0.0777 (0.1104)  time: 0.5148  data: 0.0113  max mem: 6482\n",
      "Epoch: [0]  [2300/2699]  eta: 0:03:26  lr: 0.005000  loss: 0.6169 (0.8096)  loss_classifier: 0.2570 (0.2900)  loss_box_reg: 0.2284 (0.2881)  loss_objectness: 0.0613 (0.1227)  loss_rpn_box_reg: 0.0764 (0.1089)  time: 0.5143  data: 0.0112  max mem: 6482\n",
      "Epoch: [0]  [2400/2699]  eta: 0:02:35  lr: 0.005000  loss: 0.5846 (0.8017)  loss_classifier: 0.2252 (0.2877)  loss_box_reg: 0.2340 (0.2861)  loss_objectness: 0.0504 (0.1201)  loss_rpn_box_reg: 0.0845 (0.1078)  time: 0.5185  data: 0.0116  max mem: 6482\n",
      "Epoch: [0]  [2500/2699]  eta: 0:01:43  lr: 0.005000  loss: 0.6101 (0.7945)  loss_classifier: 0.2271 (0.2857)  loss_box_reg: 0.2374 (0.2844)  loss_objectness: 0.0553 (0.1178)  loss_rpn_box_reg: 0.0721 (0.1067)  time: 0.5385  data: 0.0142  max mem: 6482\n",
      "Epoch: [0]  [2600/2699]  eta: 0:00:51  lr: 0.005000  loss: 0.6288 (0.7873)  loss_classifier: 0.2382 (0.2836)  loss_box_reg: 0.2403 (0.2826)  loss_objectness: 0.0486 (0.1155)  loss_rpn_box_reg: 0.0697 (0.1056)  time: 0.5130  data: 0.0107  max mem: 6482\n",
      "Epoch: [0]  [2698/2699]  eta: 0:00:00  lr: 0.005000  loss: 0.5862 (0.7799)  loss_classifier: 0.2229 (0.2816)  loss_box_reg: 0.2369 (0.2806)  loss_objectness: 0.0534 (0.1134)  loss_rpn_box_reg: 0.0742 (0.1044)  time: 0.5009  data: 0.0110  max mem: 6482\n",
      "Epoch: [0] Total time: 0:23:19 (0.5185 s / it)\n",
      "creating index...\n",
      "index created!\n",
      "Test:  [  0/675]  eta: 0:15:57  model_time: 0.2199 (0.2199)  evaluator_time: 0.4201 (0.4201)  time: 1.4188  data: 0.7667  max mem: 6482\n",
      "Test:  [100/675]  eta: 0:06:25  model_time: 0.1626 (0.1660)  evaluator_time: 0.3956 (0.4775)  time: 0.5691  data: 0.0112  max mem: 6482\n",
      "Test:  [200/675]  eta: 0:05:54  model_time: 0.1655 (0.1660)  evaluator_time: 0.8499 (0.5555)  time: 1.0650  data: 0.0119  max mem: 6482\n",
      "Test:  [300/675]  eta: 0:04:30  model_time: 0.1639 (0.1660)  evaluator_time: 0.5154 (0.5323)  time: 0.7620  data: 0.0114  max mem: 6482\n",
      "Test:  [400/675]  eta: 0:03:11  model_time: 0.1647 (0.1662)  evaluator_time: 0.5778 (0.5077)  time: 0.8026  data: 0.0132  max mem: 6482\n",
      "Test:  [500/675]  eta: 0:02:02  model_time: 0.1640 (0.1660)  evaluator_time: 0.9400 (0.5096)  time: 1.1397  data: 0.0123  max mem: 6482\n",
      "Test:  [600/675]  eta: 0:00:53  model_time: 0.1684 (0.1663)  evaluator_time: 0.2030 (0.5214)  time: 0.5198  data: 0.0119  max mem: 6482\n",
      "Test:  [674/675]  eta: 0:00:00  model_time: 0.1698 (0.1665)  evaluator_time: 0.2013 (0.5062)  time: 0.3877  data: 0.0113  max mem: 6482\n",
      "Test: Total time: 0:07:49 (0.6949 s / it)\n",
      "Averaged stats: model_time: 0.1698 (0.1665)  evaluator_time: 0.2013 (0.5062)\n",
      "Accumulating evaluation results...\n",
      "DONE (t=2.20s).\n",
      "IoU metric: bbox\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.402\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.863\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.308\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.017\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.387\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.471\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.015\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.137\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.481\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.087\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.468\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.549\n",
      "Epoch: [1]  [   0/2699]  eta: 1:20:43  lr: 0.005000  loss: 0.5612 (0.5612)  loss_classifier: 0.2096 (0.2096)  loss_box_reg: 0.2458 (0.2458)  loss_objectness: 0.0377 (0.0377)  loss_rpn_box_reg: 0.0681 (0.0681)  time: 1.7946  data: 1.1972  max mem: 6482\n",
      "Epoch: [1]  [ 100/2699]  eta: 0:23:06  lr: 0.005000  loss: 0.5440 (0.5638)  loss_classifier: 0.2208 (0.2171)  loss_box_reg: 0.2165 (0.2210)  loss_objectness: 0.0436 (0.0522)  loss_rpn_box_reg: 0.0749 (0.0735)  time: 0.5162  data: 0.0111  max mem: 6482\n",
      "Epoch: [1]  [ 200/2699]  eta: 0:21:59  lr: 0.005000  loss: 0.6075 (0.5734)  loss_classifier: 0.2235 (0.2201)  loss_box_reg: 0.2163 (0.2211)  loss_objectness: 0.0518 (0.0549)  loss_rpn_box_reg: 0.0678 (0.0772)  time: 0.5246  data: 0.0130  max mem: 6482\n",
      "Epoch: [1]  [ 300/2699]  eta: 0:21:00  lr: 0.005000  loss: 0.5537 (0.5712)  loss_classifier: 0.2273 (0.2196)  loss_box_reg: 0.2189 (0.2222)  loss_objectness: 0.0421 (0.0538)  loss_rpn_box_reg: 0.0696 (0.0756)  time: 0.5418  data: 0.0161  max mem: 6482\n",
      "Epoch: [1]  [ 400/2699]  eta: 0:20:02  lr: 0.005000  loss: 0.6235 (0.5742)  loss_classifier: 0.2435 (0.2213)  loss_box_reg: 0.2335 (0.2227)  loss_objectness: 0.0571 (0.0541)  loss_rpn_box_reg: 0.0791 (0.0761)  time: 0.5148  data: 0.0113  max mem: 6482\n",
      "Epoch: [1]  [ 500/2699]  eta: 0:19:09  lr: 0.005000  loss: 0.5755 (0.5760)  loss_classifier: 0.2228 (0.2219)  loss_box_reg: 0.2138 (0.2229)  loss_objectness: 0.0520 (0.0541)  loss_rpn_box_reg: 0.0806 (0.0770)  time: 0.5198  data: 0.0114  max mem: 6482\n",
      "Epoch: [1]  [ 600/2699]  eta: 0:18:16  lr: 0.005000  loss: 0.5408 (0.5733)  loss_classifier: 0.2059 (0.2212)  loss_box_reg: 0.2202 (0.2224)  loss_objectness: 0.0385 (0.0539)  loss_rpn_box_reg: 0.0634 (0.0758)  time: 0.5172  data: 0.0117  max mem: 6482\n",
      "Epoch: [1]  [ 700/2699]  eta: 0:17:23  lr: 0.005000  loss: 0.5714 (0.5744)  loss_classifier: 0.2289 (0.2220)  loss_box_reg: 0.2114 (0.2226)  loss_objectness: 0.0486 (0.0541)  loss_rpn_box_reg: 0.0746 (0.0757)  time: 0.5152  data: 0.0116  max mem: 6482\n",
      "Epoch: [1]  [ 800/2699]  eta: 0:16:30  lr: 0.005000  loss: 0.5478 (0.5711)  loss_classifier: 0.2022 (0.2207)  loss_box_reg: 0.2190 (0.2218)  loss_objectness: 0.0486 (0.0534)  loss_rpn_box_reg: 0.0616 (0.0752)  time: 0.5140  data: 0.0115  max mem: 6482\n",
      "Epoch: [1]  [ 900/2699]  eta: 0:15:38  lr: 0.005000  loss: 0.5623 (0.5689)  loss_classifier: 0.2170 (0.2204)  loss_box_reg: 0.2229 (0.2209)  loss_objectness: 0.0483 (0.0530)  loss_rpn_box_reg: 0.0650 (0.0746)  time: 0.5191  data: 0.0115  max mem: 6482\n",
      "Epoch: [1]  [1000/2699]  eta: 0:14:45  lr: 0.005000  loss: 0.6241 (0.5677)  loss_classifier: 0.2225 (0.2201)  loss_box_reg: 0.2267 (0.2208)  loss_objectness: 0.0550 (0.0527)  loss_rpn_box_reg: 0.0778 (0.0741)  time: 0.5360  data: 0.0145  max mem: 6482\n",
      "Epoch: [1]  [1100/2699]  eta: 0:13:52  lr: 0.005000  loss: 0.6067 (0.5657)  loss_classifier: 0.2307 (0.2196)  loss_box_reg: 0.2294 (0.2201)  loss_objectness: 0.0457 (0.0523)  loss_rpn_box_reg: 0.0711 (0.0737)  time: 0.5282  data: 0.0147  max mem: 6482\n",
      "Epoch: [1]  [1200/2699]  eta: 0:13:00  lr: 0.005000  loss: 0.5369 (0.5629)  loss_classifier: 0.2187 (0.2187)  loss_box_reg: 0.2097 (0.2191)  loss_objectness: 0.0429 (0.0519)  loss_rpn_box_reg: 0.0672 (0.0732)  time: 0.5149  data: 0.0111  max mem: 6482\n",
      "Epoch: [1]  [1300/2699]  eta: 0:12:08  lr: 0.005000  loss: 0.5730 (0.5596)  loss_classifier: 0.2237 (0.2175)  loss_box_reg: 0.2057 (0.2178)  loss_objectness: 0.0455 (0.0513)  loss_rpn_box_reg: 0.0835 (0.0729)  time: 0.5197  data: 0.0127  max mem: 6482\n",
      "Epoch: [1]  [1400/2699]  eta: 0:11:16  lr: 0.005000  loss: 0.5408 (0.5576)  loss_classifier: 0.2065 (0.2166)  loss_box_reg: 0.2167 (0.2169)  loss_objectness: 0.0431 (0.0512)  loss_rpn_box_reg: 0.0708 (0.0728)  time: 0.5149  data: 0.0113  max mem: 6482\n",
      "Epoch: [1]  [1500/2699]  eta: 0:10:24  lr: 0.005000  loss: 0.5149 (0.5570)  loss_classifier: 0.2203 (0.2166)  loss_box_reg: 0.2050 (0.2167)  loss_objectness: 0.0428 (0.0512)  loss_rpn_box_reg: 0.0689 (0.0726)  time: 0.5198  data: 0.0122  max mem: 6482\n",
      "Epoch: [1]  [1600/2699]  eta: 0:09:40  lr: 0.005000  loss: 0.5144 (0.5558)  loss_classifier: 0.1833 (0.2163)  loss_box_reg: 0.2077 (0.2161)  loss_objectness: 0.0419 (0.0510)  loss_rpn_box_reg: 0.0596 (0.0724)  time: 0.5162  data: 0.0118  max mem: 6482\n",
      "Epoch: [1]  [1700/2699]  eta: 0:08:46  lr: 0.005000  loss: 0.5156 (0.5560)  loss_classifier: 0.1988 (0.2165)  loss_box_reg: 0.2081 (0.2160)  loss_objectness: 0.0365 (0.0509)  loss_rpn_box_reg: 0.0697 (0.0725)  time: 0.5151  data: 0.0115  max mem: 6482\n",
      "Epoch: [1]  [1800/2699]  eta: 0:07:53  lr: 0.005000  loss: 0.5291 (0.5539)  loss_classifier: 0.1999 (0.2157)  loss_box_reg: 0.2036 (0.2154)  loss_objectness: 0.0481 (0.0507)  loss_rpn_box_reg: 0.0694 (0.0721)  time: 0.5159  data: 0.0119  max mem: 6482\n",
      "Epoch: [1]  [1900/2699]  eta: 0:07:00  lr: 0.005000  loss: 0.5093 (0.5525)  loss_classifier: 0.2019 (0.2151)  loss_box_reg: 0.2015 (0.2148)  loss_objectness: 0.0364 (0.0504)  loss_rpn_box_reg: 0.0617 (0.0722)  time: 0.5413  data: 0.0140  max mem: 6482\n",
      "Epoch: [1]  [2000/2699]  eta: 0:06:07  lr: 0.005000  loss: 0.4979 (0.5507)  loss_classifier: 0.1940 (0.2145)  loss_box_reg: 0.1948 (0.2140)  loss_objectness: 0.0430 (0.0503)  loss_rpn_box_reg: 0.0661 (0.0719)  time: 0.5234  data: 0.0121  max mem: 6482\n",
      "Epoch: [1]  [2100/2699]  eta: 0:05:14  lr: 0.005000  loss: 0.5147 (0.5494)  loss_classifier: 0.2074 (0.2141)  loss_box_reg: 0.1876 (0.2133)  loss_objectness: 0.0497 (0.0501)  loss_rpn_box_reg: 0.0661 (0.0718)  time: 0.5163  data: 0.0115  max mem: 6482\n",
      "Epoch: [1]  [2200/2699]  eta: 0:04:22  lr: 0.005000  loss: 0.4715 (0.5488)  loss_classifier: 0.1973 (0.2140)  loss_box_reg: 0.1817 (0.2130)  loss_objectness: 0.0427 (0.0500)  loss_rpn_box_reg: 0.0575 (0.0718)  time: 0.5155  data: 0.0115  max mem: 6482\n",
      "Epoch: [1]  [2300/2699]  eta: 0:03:29  lr: 0.005000  loss: 0.5202 (0.5480)  loss_classifier: 0.2174 (0.2137)  loss_box_reg: 0.1999 (0.2126)  loss_objectness: 0.0504 (0.0499)  loss_rpn_box_reg: 0.0663 (0.0719)  time: 0.5169  data: 0.0112  max mem: 6482\n",
      "Epoch: [1]  [2400/2699]  eta: 0:02:37  lr: 0.005000  loss: 0.5695 (0.5478)  loss_classifier: 0.2144 (0.2137)  loss_box_reg: 0.2177 (0.2124)  loss_objectness: 0.0418 (0.0499)  loss_rpn_box_reg: 0.0639 (0.0718)  time: 0.5181  data: 0.0115  max mem: 6482\n",
      "Epoch: [1]  [2500/2699]  eta: 0:01:44  lr: 0.005000  loss: 0.4940 (0.5465)  loss_classifier: 0.2062 (0.2134)  loss_box_reg: 0.1909 (0.2120)  loss_objectness: 0.0436 (0.0496)  loss_rpn_box_reg: 0.0631 (0.0715)  time: 0.5166  data: 0.0115  max mem: 6482\n",
      "Epoch: [1]  [2600/2699]  eta: 0:00:51  lr: 0.005000  loss: 0.5057 (0.5455)  loss_classifier: 0.2002 (0.2131)  loss_box_reg: 0.2010 (0.2115)  loss_objectness: 0.0372 (0.0495)  loss_rpn_box_reg: 0.0689 (0.0714)  time: 0.5316  data: 0.0133  max mem: 6482\n",
      "Epoch: [1]  [2698/2699]  eta: 0:00:00  lr: 0.005000  loss: 0.4822 (0.5443)  loss_classifier: 0.1862 (0.2128)  loss_box_reg: 0.1934 (0.2111)  loss_objectness: 0.0350 (0.0493)  loss_rpn_box_reg: 0.0509 (0.0712)  time: 0.5032  data: 0.0114  max mem: 6482\n",
      "Epoch: [1] Total time: 0:23:35 (0.5245 s / it)\n",
      "creating index...\n",
      "index created!\n",
      "Test:  [  0/675]  eta: 0:15:12  model_time: 0.2265 (0.2265)  evaluator_time: 0.4025 (0.4025)  time: 1.3524  data: 0.6972  max mem: 6482\n",
      "Test:  [100/675]  eta: 0:05:32  model_time: 0.1552 (0.1579)  evaluator_time: 0.3193 (0.3927)  time: 0.4893  data: 0.0117  max mem: 6482\n",
      "Test:  [200/675]  eta: 0:05:17  model_time: 0.1574 (0.1581)  evaluator_time: 0.7556 (0.4854)  time: 0.9878  data: 0.0122  max mem: 6482\n",
      "Test:  [300/675]  eta: 0:04:01  model_time: 0.1575 (0.1586)  evaluator_time: 0.4572 (0.4615)  time: 0.6380  data: 0.0129  max mem: 6482\n",
      "Test:  [400/675]  eta: 0:02:48  model_time: 0.1555 (0.1586)  evaluator_time: 0.4649 (0.4328)  time: 0.6564  data: 0.0117  max mem: 6482\n",
      "Test:  [500/675]  eta: 0:01:47  model_time: 0.1555 (0.1586)  evaluator_time: 0.8496 (0.4341)  time: 1.0451  data: 0.0121  max mem: 6482\n",
      "Test:  [600/675]  eta: 0:00:47  model_time: 0.1629 (0.1590)  evaluator_time: 0.1905 (0.4495)  time: 0.5124  data: 0.0133  max mem: 6482\n",
      "Test:  [674/675]  eta: 0:00:00  model_time: 0.1635 (0.1590)  evaluator_time: 0.1195 (0.4324)  time: 0.3096  data: 0.0114  max mem: 6482\n",
      "Test: Total time: 0:06:54 (0.6140 s / it)\n",
      "Averaged stats: model_time: 0.1635 (0.1590)  evaluator_time: 0.1195 (0.4324)\n",
      "Accumulating evaluation results...\n",
      "DONE (t=1.86s).\n",
      "IoU metric: bbox\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.396\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.858\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.300\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.034\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.384\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.473\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.015\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.135\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.475\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.099\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.455\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.567\n",
      "Epoch: [2]  [   0/2699]  eta: 1:14:48  lr: 0.005000  loss: 0.5442 (0.5442)  loss_classifier: 0.2067 (0.2067)  loss_box_reg: 0.2176 (0.2176)  loss_objectness: 0.0430 (0.0430)  loss_rpn_box_reg: 0.0768 (0.0768)  time: 1.6630  data: 0.9859  max mem: 6482\n",
      "Epoch: [2]  [ 100/2699]  eta: 0:23:10  lr: 0.005000  loss: 0.4880 (0.5069)  loss_classifier: 0.1836 (0.1989)  loss_box_reg: 0.2021 (0.1951)  loss_objectness: 0.0354 (0.0421)  loss_rpn_box_reg: 0.0646 (0.0709)  time: 0.5211  data: 0.0114  max mem: 6482\n",
      "Epoch: [2]  [ 200/2699]  eta: 0:21:56  lr: 0.005000  loss: 0.4742 (0.4972)  loss_classifier: 0.1903 (0.1964)  loss_box_reg: 0.1928 (0.1929)  loss_objectness: 0.0281 (0.0407)  loss_rpn_box_reg: 0.0575 (0.0673)  time: 0.5155  data: 0.0114  max mem: 6482\n",
      "Epoch: [2]  [ 300/2699]  eta: 0:20:57  lr: 0.005000  loss: 0.4859 (0.4955)  loss_classifier: 0.1984 (0.1963)  loss_box_reg: 0.1938 (0.1918)  loss_objectness: 0.0452 (0.0411)  loss_rpn_box_reg: 0.0555 (0.0663)  time: 0.5173  data: 0.0117  max mem: 6482\n",
      "Epoch: [2]  [ 400/2699]  eta: 0:20:02  lr: 0.005000  loss: 0.4748 (0.4960)  loss_classifier: 0.1869 (0.1971)  loss_box_reg: 0.1835 (0.1913)  loss_objectness: 0.0357 (0.0412)  loss_rpn_box_reg: 0.0629 (0.0664)  time: 0.5240  data: 0.0121  max mem: 6482\n",
      "Epoch: [2]  [ 500/2699]  eta: 0:19:08  lr: 0.005000  loss: 0.5193 (0.4951)  loss_classifier: 0.2156 (0.1967)  loss_box_reg: 0.1992 (0.1909)  loss_objectness: 0.0381 (0.0414)  loss_rpn_box_reg: 0.0623 (0.0662)  time: 0.5347  data: 0.0152  max mem: 6482\n",
      "Epoch: [2]  [ 600/2699]  eta: 0:18:14  lr: 0.005000  loss: 0.5560 (0.4978)  loss_classifier: 0.2151 (0.1975)  loss_box_reg: 0.2041 (0.1918)  loss_objectness: 0.0451 (0.0415)  loss_rpn_box_reg: 0.0703 (0.0670)  time: 0.5167  data: 0.0127  max mem: 6482\n",
      "Epoch: [2]  [ 700/2699]  eta: 0:17:22  lr: 0.005000  loss: 0.4875 (0.4943)  loss_classifier: 0.1833 (0.1957)  loss_box_reg: 0.1824 (0.1911)  loss_objectness: 0.0423 (0.0412)  loss_rpn_box_reg: 0.0653 (0.0663)  time: 0.5149  data: 0.0113  max mem: 6482\n",
      "Epoch: [2]  [ 800/2699]  eta: 0:16:29  lr: 0.005000  loss: 0.4852 (0.4926)  loss_classifier: 0.1902 (0.1951)  loss_box_reg: 0.1860 (0.1906)  loss_objectness: 0.0339 (0.0410)  loss_rpn_box_reg: 0.0676 (0.0659)  time: 0.5149  data: 0.0113  max mem: 6482\n",
      "Epoch: [2]  [ 900/2699]  eta: 0:15:37  lr: 0.005000  loss: 0.5250 (0.4934)  loss_classifier: 0.2003 (0.1956)  loss_box_reg: 0.1991 (0.1907)  loss_objectness: 0.0449 (0.0410)  loss_rpn_box_reg: 0.0693 (0.0661)  time: 0.5189  data: 0.0114  max mem: 6482\n",
      "Epoch: [2]  [1000/2699]  eta: 0:14:45  lr: 0.005000  loss: 0.4918 (0.4938)  loss_classifier: 0.1851 (0.1958)  loss_box_reg: 0.2013 (0.1911)  loss_objectness: 0.0447 (0.0410)  loss_rpn_box_reg: 0.0585 (0.0659)  time: 0.5145  data: 0.0113  max mem: 6482\n",
      "Epoch: [2]  [1100/2699]  eta: 0:13:52  lr: 0.005000  loss: 0.5050 (0.4921)  loss_classifier: 0.1913 (0.1950)  loss_box_reg: 0.1830 (0.1907)  loss_objectness: 0.0305 (0.0406)  loss_rpn_box_reg: 0.0602 (0.0657)  time: 0.5166  data: 0.0119  max mem: 6482\n",
      "Epoch: [2]  [1200/2699]  eta: 0:13:00  lr: 0.005000  loss: 0.5182 (0.4930)  loss_classifier: 0.1951 (0.1951)  loss_box_reg: 0.1965 (0.1909)  loss_objectness: 0.0459 (0.0408)  loss_rpn_box_reg: 0.0719 (0.0661)  time: 0.5345  data: 0.0136  max mem: 6482\n",
      "Epoch: [2]  [1300/2699]  eta: 0:12:08  lr: 0.005000  loss: 0.4448 (0.4916)  loss_classifier: 0.1687 (0.1947)  loss_box_reg: 0.1757 (0.1905)  loss_objectness: 0.0292 (0.0406)  loss_rpn_box_reg: 0.0490 (0.0658)  time: 0.5267  data: 0.0130  max mem: 6482\n",
      "Epoch: [2]  [1400/2699]  eta: 0:11:15  lr: 0.005000  loss: 0.4998 (0.4910)  loss_classifier: 0.2003 (0.1943)  loss_box_reg: 0.1862 (0.1906)  loss_objectness: 0.0377 (0.0405)  loss_rpn_box_reg: 0.0620 (0.0656)  time: 0.5177  data: 0.0116  max mem: 6482\n",
      "Epoch: [2]  [1500/2699]  eta: 0:10:23  lr: 0.005000  loss: 0.4905 (0.4908)  loss_classifier: 0.1971 (0.1942)  loss_box_reg: 0.1862 (0.1906)  loss_objectness: 0.0348 (0.0404)  loss_rpn_box_reg: 0.0635 (0.0656)  time: 0.5161  data: 0.0118  max mem: 6482\n",
      "Epoch: [2]  [1600/2699]  eta: 0:09:31  lr: 0.005000  loss: 0.4557 (0.4901)  loss_classifier: 0.1788 (0.1938)  loss_box_reg: 0.1868 (0.1904)  loss_objectness: 0.0324 (0.0403)  loss_rpn_box_reg: 0.0623 (0.0656)  time: 0.5150  data: 0.0117  max mem: 6482\n",
      "Epoch: [2]  [1700/2699]  eta: 0:08:39  lr: 0.005000  loss: 0.5181 (0.4899)  loss_classifier: 0.2059 (0.1938)  loss_box_reg: 0.1989 (0.1904)  loss_objectness: 0.0423 (0.0403)  loss_rpn_box_reg: 0.0804 (0.0655)  time: 0.5216  data: 0.0127  max mem: 6482\n",
      "Epoch: [2]  [1800/2699]  eta: 0:07:47  lr: 0.005000  loss: 0.4535 (0.4888)  loss_classifier: 0.1833 (0.1932)  loss_box_reg: 0.1775 (0.1901)  loss_objectness: 0.0273 (0.0402)  loss_rpn_box_reg: 0.0615 (0.0653)  time: 0.5132  data: 0.0111  max mem: 6482\n",
      "Epoch: [2]  [1900/2699]  eta: 0:06:55  lr: 0.005000  loss: 0.4598 (0.4896)  loss_classifier: 0.1751 (0.1935)  loss_box_reg: 0.1649 (0.1902)  loss_objectness: 0.0393 (0.0404)  loss_rpn_box_reg: 0.0629 (0.0655)  time: 0.5233  data: 0.0121  max mem: 6482\n",
      "Epoch: [2]  [2000/2699]  eta: 0:06:03  lr: 0.005000  loss: 0.5186 (0.4892)  loss_classifier: 0.2016 (0.1934)  loss_box_reg: 0.1966 (0.1902)  loss_objectness: 0.0336 (0.0402)  loss_rpn_box_reg: 0.0692 (0.0654)  time: 0.5350  data: 0.0126  max mem: 6482\n",
      "Epoch: [2]  [2100/2699]  eta: 0:05:11  lr: 0.005000  loss: 0.4867 (0.4892)  loss_classifier: 0.1923 (0.1933)  loss_box_reg: 0.1999 (0.1903)  loss_objectness: 0.0372 (0.0402)  loss_rpn_box_reg: 0.0622 (0.0653)  time: 0.5140  data: 0.0114  max mem: 6482\n",
      "Epoch: [2]  [2200/2699]  eta: 0:04:19  lr: 0.005000  loss: 0.4938 (0.4893)  loss_classifier: 0.1974 (0.1933)  loss_box_reg: 0.1892 (0.1902)  loss_objectness: 0.0464 (0.0404)  loss_rpn_box_reg: 0.0646 (0.0653)  time: 0.5192  data: 0.0116  max mem: 6482\n",
      "Epoch: [2]  [2300/2699]  eta: 0:03:27  lr: 0.005000  loss: 0.4393 (0.4879)  loss_classifier: 0.1667 (0.1927)  loss_box_reg: 0.1839 (0.1898)  loss_objectness: 0.0337 (0.0403)  loss_rpn_box_reg: 0.0542 (0.0651)  time: 0.5179  data: 0.0122  max mem: 6482\n",
      "Epoch: [2]  [2400/2699]  eta: 0:02:35  lr: 0.005000  loss: 0.4906 (0.4872)  loss_classifier: 0.1881 (0.1924)  loss_box_reg: 0.1828 (0.1895)  loss_objectness: 0.0478 (0.0403)  loss_rpn_box_reg: 0.0637 (0.0650)  time: 0.5153  data: 0.0115  max mem: 6482\n",
      "Epoch: [2]  [2500/2699]  eta: 0:01:43  lr: 0.005000  loss: 0.4863 (0.4874)  loss_classifier: 0.2034 (0.1926)  loss_box_reg: 0.1799 (0.1894)  loss_objectness: 0.0370 (0.0404)  loss_rpn_box_reg: 0.0763 (0.0650)  time: 0.5171  data: 0.0112  max mem: 6482\n",
      "Epoch: [2]  [2600/2699]  eta: 0:00:51  lr: 0.005000  loss: 0.4487 (0.4872)  loss_classifier: 0.1945 (0.1927)  loss_box_reg: 0.1671 (0.1893)  loss_objectness: 0.0342 (0.0403)  loss_rpn_box_reg: 0.0608 (0.0649)  time: 0.5171  data: 0.0118  max mem: 6482\n",
      "Epoch: [2]  [2698/2699]  eta: 0:00:00  lr: 0.005000  loss: 0.4255 (0.4861)  loss_classifier: 0.1754 (0.1922)  loss_box_reg: 0.1753 (0.1890)  loss_objectness: 0.0259 (0.0403)  loss_rpn_box_reg: 0.0437 (0.0647)  time: 0.5100  data: 0.0149  max mem: 6482\n",
      "Epoch: [2] Total time: 0:23:23 (0.5199 s / it)\n",
      "creating index...\n",
      "index created!\n",
      "Test:  [  0/675]  eta: 0:15:17  model_time: 0.2378 (0.2378)  evaluator_time: 0.3293 (0.3293)  time: 1.3598  data: 0.7686  max mem: 6482\n",
      "Test:  [100/675]  eta: 0:05:49  model_time: 0.1521 (0.1567)  evaluator_time: 0.3656 (0.4232)  time: 0.5165  data: 0.0117  max mem: 6482\n",
      "Test:  [200/675]  eta: 0:05:29  model_time: 0.1613 (0.1576)  evaluator_time: 0.7736 (0.5110)  time: 1.0309  data: 0.0119  max mem: 6482\n",
      "Test:  [300/675]  eta: 0:04:08  model_time: 0.1552 (0.1578)  evaluator_time: 0.4090 (0.4820)  time: 0.6542  data: 0.0118  max mem: 6482\n",
      "Test:  [400/675]  eta: 0:02:55  model_time: 0.1555 (0.1577)  evaluator_time: 0.5367 (0.4580)  time: 0.7557  data: 0.0118  max mem: 6482\n",
      "Test:  [500/675]  eta: 0:01:52  model_time: 0.1581 (0.1573)  evaluator_time: 0.9083 (0.4610)  time: 1.1101  data: 0.0127  max mem: 6482\n",
      "Test:  [600/675]  eta: 0:00:49  model_time: 0.1618 (0.1578)  evaluator_time: 0.1750 (0.4735)  time: 0.4775  data: 0.0115  max mem: 6482\n",
      "Test:  [674/675]  eta: 0:00:00  model_time: 0.1570 (0.1577)  evaluator_time: 0.1517 (0.4560)  time: 0.3484  data: 0.0147  max mem: 6482\n",
      "Test: Total time: 0:07:09 (0.6363 s / it)\n",
      "Averaged stats: model_time: 0.1570 (0.1577)  evaluator_time: 0.1517 (0.4560)\n",
      "Accumulating evaluation results...\n",
      "DONE (t=1.98s).\n",
      "IoU metric: bbox\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.418\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.881\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.332\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.032\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.399\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.505\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.015\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.138\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.492\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.102\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.471\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.591\n",
      "Epoch: [3]  [   0/2699]  eta: 1:41:53  lr: 0.000500  loss: 0.3996 (0.3996)  loss_classifier: 0.1623 (0.1623)  loss_box_reg: 0.1406 (0.1406)  loss_objectness: 0.0291 (0.0291)  loss_rpn_box_reg: 0.0676 (0.0676)  time: 2.2652  data: 1.3396  max mem: 6482\n",
      "Epoch: [3]  [ 100/2699]  eta: 0:23:10  lr: 0.000500  loss: 0.4645 (0.4293)  loss_classifier: 0.1886 (0.1732)  loss_box_reg: 0.1725 (0.1641)  loss_objectness: 0.0381 (0.0347)  loss_rpn_box_reg: 0.0541 (0.0573)  time: 0.5151  data: 0.0113  max mem: 6482\n",
      "Epoch: [3]  [ 200/2699]  eta: 0:21:58  lr: 0.000500  loss: 0.4364 (0.4213)  loss_classifier: 0.1676 (0.1696)  loss_box_reg: 0.1583 (0.1613)  loss_objectness: 0.0304 (0.0337)  loss_rpn_box_reg: 0.0558 (0.0567)  time: 0.5154  data: 0.0121  max mem: 6482\n",
      "Epoch: [3]  [ 300/2699]  eta: 0:20:57  lr: 0.000500  loss: 0.4172 (0.4216)  loss_classifier: 0.1782 (0.1695)  loss_box_reg: 0.1616 (0.1622)  loss_objectness: 0.0308 (0.0340)  loss_rpn_box_reg: 0.0505 (0.0559)  time: 0.5150  data: 0.0116  max mem: 6482\n",
      "Epoch: [3]  [ 400/2699]  eta: 0:20:03  lr: 0.000500  loss: 0.4136 (0.4180)  loss_classifier: 0.1677 (0.1681)  loss_box_reg: 0.1602 (0.1618)  loss_objectness: 0.0311 (0.0330)  loss_rpn_box_reg: 0.0538 (0.0552)  time: 0.5143  data: 0.0116  max mem: 6482\n",
      "Epoch: [3]  [ 500/2699]  eta: 0:19:09  lr: 0.000500  loss: 0.4570 (0.4195)  loss_classifier: 0.1794 (0.1685)  loss_box_reg: 0.1709 (0.1620)  loss_objectness: 0.0310 (0.0331)  loss_rpn_box_reg: 0.0600 (0.0559)  time: 0.5161  data: 0.0117  max mem: 6482\n",
      "Epoch: [3]  [ 600/2699]  eta: 0:18:16  lr: 0.000500  loss: 0.4564 (0.4181)  loss_classifier: 0.1842 (0.1683)  loss_box_reg: 0.1749 (0.1619)  loss_objectness: 0.0344 (0.0326)  loss_rpn_box_reg: 0.0526 (0.0553)  time: 0.5178  data: 0.0118  max mem: 6482\n",
      "Epoch: [3]  [ 700/2699]  eta: 0:17:23  lr: 0.000500  loss: 0.4402 (0.4197)  loss_classifier: 0.1695 (0.1690)  loss_box_reg: 0.1614 (0.1625)  loss_objectness: 0.0306 (0.0328)  loss_rpn_box_reg: 0.0543 (0.0553)  time: 0.5367  data: 0.0145  max mem: 6482\n",
      "Epoch: [3]  [ 800/2699]  eta: 0:16:30  lr: 0.000500  loss: 0.3811 (0.4181)  loss_classifier: 0.1547 (0.1685)  loss_box_reg: 0.1536 (0.1622)  loss_objectness: 0.0230 (0.0324)  loss_rpn_box_reg: 0.0532 (0.0550)  time: 0.5264  data: 0.0117  max mem: 6482\n",
      "Epoch: [3]  [ 900/2699]  eta: 0:15:37  lr: 0.000500  loss: 0.3733 (0.4161)  loss_classifier: 0.1594 (0.1678)  loss_box_reg: 0.1466 (0.1616)  loss_objectness: 0.0230 (0.0320)  loss_rpn_box_reg: 0.0460 (0.0548)  time: 0.5137  data: 0.0116  max mem: 6482\n",
      "Epoch: [3]  [1000/2699]  eta: 0:14:45  lr: 0.000500  loss: 0.4256 (0.4176)  loss_classifier: 0.1731 (0.1685)  loss_box_reg: 0.1524 (0.1618)  loss_objectness: 0.0293 (0.0321)  loss_rpn_box_reg: 0.0586 (0.0551)  time: 0.5153  data: 0.0114  max mem: 6482\n",
      "Epoch: [3]  [1100/2699]  eta: 0:13:52  lr: 0.000500  loss: 0.4242 (0.4176)  loss_classifier: 0.1651 (0.1685)  loss_box_reg: 0.1662 (0.1619)  loss_objectness: 0.0313 (0.0321)  loss_rpn_box_reg: 0.0521 (0.0551)  time: 0.5135  data: 0.0111  max mem: 6482\n",
      "Epoch: [3]  [1200/2699]  eta: 0:13:00  lr: 0.000500  loss: 0.4110 (0.4171)  loss_classifier: 0.1771 (0.1684)  loss_box_reg: 0.1607 (0.1617)  loss_objectness: 0.0248 (0.0320)  loss_rpn_box_reg: 0.0537 (0.0550)  time: 0.5188  data: 0.0155  max mem: 6482\n",
      "Epoch: [3]  [1300/2699]  eta: 0:12:08  lr: 0.000500  loss: 0.3927 (0.4171)  loss_classifier: 0.1563 (0.1683)  loss_box_reg: 0.1545 (0.1616)  loss_objectness: 0.0209 (0.0320)  loss_rpn_box_reg: 0.0438 (0.0551)  time: 0.5148  data: 0.0119  max mem: 6482\n",
      "Epoch: [3]  [1400/2699]  eta: 0:11:16  lr: 0.000500  loss: 0.4288 (0.4171)  loss_classifier: 0.1808 (0.1684)  loss_box_reg: 0.1643 (0.1618)  loss_objectness: 0.0289 (0.0319)  loss_rpn_box_reg: 0.0539 (0.0551)  time: 0.5260  data: 0.0129  max mem: 6482\n",
      "Epoch: [3]  [1500/2699]  eta: 0:10:24  lr: 0.000500  loss: 0.3925 (0.4164)  loss_classifier: 0.1531 (0.1682)  loss_box_reg: 0.1384 (0.1616)  loss_objectness: 0.0272 (0.0316)  loss_rpn_box_reg: 0.0620 (0.0550)  time: 0.5351  data: 0.0152  max mem: 6482\n",
      "Epoch: [3]  [1600/2699]  eta: 0:09:32  lr: 0.000500  loss: 0.4041 (0.4160)  loss_classifier: 0.1599 (0.1681)  loss_box_reg: 0.1615 (0.1614)  loss_objectness: 0.0307 (0.0315)  loss_rpn_box_reg: 0.0545 (0.0550)  time: 0.5147  data: 0.0115  max mem: 6482\n",
      "Epoch: [3]  [1700/2699]  eta: 0:08:40  lr: 0.000500  loss: 0.4521 (0.4157)  loss_classifier: 0.1823 (0.1679)  loss_box_reg: 0.1760 (0.1613)  loss_objectness: 0.0314 (0.0315)  loss_rpn_box_reg: 0.0576 (0.0550)  time: 0.5169  data: 0.0115  max mem: 6482\n",
      "Epoch: [3]  [1800/2699]  eta: 0:07:47  lr: 0.000500  loss: 0.3769 (0.4148)  loss_classifier: 0.1491 (0.1676)  loss_box_reg: 0.1385 (0.1609)  loss_objectness: 0.0270 (0.0313)  loss_rpn_box_reg: 0.0487 (0.0549)  time: 0.5148  data: 0.0112  max mem: 6482\n",
      "Epoch: [3]  [1900/2699]  eta: 0:06:55  lr: 0.000500  loss: 0.4072 (0.4154)  loss_classifier: 0.1607 (0.1680)  loss_box_reg: 0.1532 (0.1611)  loss_objectness: 0.0224 (0.0313)  loss_rpn_box_reg: 0.0560 (0.0551)  time: 0.5164  data: 0.0116  max mem: 6482\n",
      "Epoch: [3]  [2000/2699]  eta: 0:06:03  lr: 0.000500  loss: 0.4148 (0.4144)  loss_classifier: 0.1613 (0.1675)  loss_box_reg: 0.1613 (0.1608)  loss_objectness: 0.0283 (0.0312)  loss_rpn_box_reg: 0.0589 (0.0549)  time: 0.5184  data: 0.0118  max mem: 6482\n",
      "Epoch: [3]  [2100/2699]  eta: 0:05:11  lr: 0.000500  loss: 0.3714 (0.4142)  loss_classifier: 0.1449 (0.1675)  loss_box_reg: 0.1482 (0.1607)  loss_objectness: 0.0216 (0.0312)  loss_rpn_box_reg: 0.0490 (0.0549)  time: 0.5198  data: 0.0120  max mem: 6482\n",
      "Epoch: [3]  [2200/2699]  eta: 0:04:19  lr: 0.000500  loss: 0.3639 (0.4134)  loss_classifier: 0.1576 (0.1672)  loss_box_reg: 0.1496 (0.1603)  loss_objectness: 0.0248 (0.0311)  loss_rpn_box_reg: 0.0448 (0.0548)  time: 0.5330  data: 0.0171  max mem: 6482\n",
      "Epoch: [3]  [2300/2699]  eta: 0:03:27  lr: 0.000500  loss: 0.4244 (0.4131)  loss_classifier: 0.1744 (0.1672)  loss_box_reg: 0.1624 (0.1602)  loss_objectness: 0.0285 (0.0311)  loss_rpn_box_reg: 0.0544 (0.0547)  time: 0.5298  data: 0.0128  max mem: 6482\n",
      "Epoch: [3]  [2400/2699]  eta: 0:02:35  lr: 0.000500  loss: 0.3524 (0.4123)  loss_classifier: 0.1576 (0.1669)  loss_box_reg: 0.1422 (0.1599)  loss_objectness: 0.0217 (0.0310)  loss_rpn_box_reg: 0.0433 (0.0545)  time: 0.5168  data: 0.0116  max mem: 6482\n",
      "Epoch: [3]  [2500/2699]  eta: 0:01:43  lr: 0.000500  loss: 0.3810 (0.4115)  loss_classifier: 0.1559 (0.1665)  loss_box_reg: 0.1491 (0.1598)  loss_objectness: 0.0264 (0.0309)  loss_rpn_box_reg: 0.0537 (0.0543)  time: 0.5159  data: 0.0116  max mem: 6482\n",
      "Epoch: [3]  [2600/2699]  eta: 0:00:51  lr: 0.000500  loss: 0.3862 (0.4113)  loss_classifier: 0.1565 (0.1665)  loss_box_reg: 0.1434 (0.1597)  loss_objectness: 0.0221 (0.0308)  loss_rpn_box_reg: 0.0506 (0.0543)  time: 0.5147  data: 0.0114  max mem: 6482\n",
      "Epoch: [3]  [2698/2699]  eta: 0:00:00  lr: 0.000500  loss: 0.4083 (0.4112)  loss_classifier: 0.1726 (0.1664)  loss_box_reg: 0.1591 (0.1596)  loss_objectness: 0.0313 (0.0308)  loss_rpn_box_reg: 0.0644 (0.0544)  time: 0.5013  data: 0.0113  max mem: 6482\n",
      "Epoch: [3] Total time: 0:23:24 (0.5203 s / it)\n",
      "creating index...\n",
      "index created!\n",
      "Test:  [  0/675]  eta: 0:14:57  model_time: 0.2494 (0.2494)  evaluator_time: 0.4244 (0.4244)  time: 1.3295  data: 0.6283  max mem: 6482\n",
      "Test:  [100/675]  eta: 0:05:18  model_time: 0.1505 (0.1538)  evaluator_time: 0.2854 (0.3719)  time: 0.4998  data: 0.0138  max mem: 6482\n",
      "Test:  [200/675]  eta: 0:05:06  model_time: 0.1523 (0.1535)  evaluator_time: 0.7102 (0.4681)  time: 0.9618  data: 0.0115  max mem: 6482\n",
      "Test:  [300/675]  eta: 0:03:53  model_time: 0.1506 (0.1540)  evaluator_time: 0.4087 (0.4452)  time: 0.5962  data: 0.0113  max mem: 6482\n",
      "Test:  [400/675]  eta: 0:02:43  model_time: 0.1504 (0.1542)  evaluator_time: 0.5006 (0.4170)  time: 0.6766  data: 0.0141  max mem: 6482\n",
      "Test:  [500/675]  eta: 0:01:44  model_time: 0.1509 (0.1539)  evaluator_time: 0.8844 (0.4183)  time: 1.0758  data: 0.0124  max mem: 6482\n",
      "Test:  [600/675]  eta: 0:00:45  model_time: 0.1586 (0.1543)  evaluator_time: 0.1681 (0.4330)  time: 0.4607  data: 0.0119  max mem: 6482\n",
      "Test:  [674/675]  eta: 0:00:00  model_time: 0.1594 (0.1543)  evaluator_time: 0.0935 (0.4159)  time: 0.2743  data: 0.0110  max mem: 6482\n",
      "Test: Total time: 0:06:40 (0.5927 s / it)\n",
      "Averaged stats: model_time: 0.1594 (0.1543)  evaluator_time: 0.0935 (0.4159)\n",
      "Accumulating evaluation results...\n",
      "DONE (t=1.74s).\n",
      "IoU metric: bbox\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.467\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.899\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.429\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.053\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.457\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.526\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.016\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.149\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.536\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.146\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.527\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.591\n",
      "Epoch: [4]  [   0/2699]  eta: 1:12:54  lr: 0.000500  loss: 0.3393 (0.3393)  loss_classifier: 0.1475 (0.1475)  loss_box_reg: 0.1255 (0.1255)  loss_objectness: 0.0211 (0.0211)  loss_rpn_box_reg: 0.0452 (0.0452)  time: 1.6206  data: 0.9799  max mem: 6482\n",
      "Epoch: [4]  [ 100/2699]  eta: 0:23:03  lr: 0.000500  loss: 0.3683 (0.3887)  loss_classifier: 0.1562 (0.1564)  loss_box_reg: 0.1498 (0.1538)  loss_objectness: 0.0237 (0.0281)  loss_rpn_box_reg: 0.0489 (0.0505)  time: 0.5202  data: 0.0129  max mem: 6482\n",
      "Epoch: [4]  [ 200/2699]  eta: 0:21:55  lr: 0.000500  loss: 0.3958 (0.3913)  loss_classifier: 0.1554 (0.1568)  loss_box_reg: 0.1594 (0.1548)  loss_objectness: 0.0258 (0.0277)  loss_rpn_box_reg: 0.0543 (0.0520)  time: 0.5152  data: 0.0113  max mem: 6482\n",
      "Epoch: [4]  [ 300/2699]  eta: 0:20:58  lr: 0.000500  loss: 0.3613 (0.3917)  loss_classifier: 0.1465 (0.1571)  loss_box_reg: 0.1564 (0.1549)  loss_objectness: 0.0215 (0.0275)  loss_rpn_box_reg: 0.0510 (0.0523)  time: 0.5184  data: 0.0120  max mem: 6482\n",
      "Epoch: [4]  [ 400/2699]  eta: 0:20:04  lr: 0.000500  loss: 0.4420 (0.3959)  loss_classifier: 0.1638 (0.1589)  loss_box_reg: 0.1642 (0.1558)  loss_objectness: 0.0247 (0.0280)  loss_rpn_box_reg: 0.0500 (0.0531)  time: 0.5162  data: 0.0113  max mem: 6482\n",
      "Epoch: [4]  [ 500/2699]  eta: 0:19:10  lr: 0.000500  loss: 0.3933 (0.3956)  loss_classifier: 0.1656 (0.1589)  loss_box_reg: 0.1494 (0.1559)  loss_objectness: 0.0237 (0.0279)  loss_rpn_box_reg: 0.0524 (0.0530)  time: 0.5150  data: 0.0114  max mem: 6482\n",
      "Epoch: [4]  [ 600/2699]  eta: 0:18:17  lr: 0.000500  loss: 0.3847 (0.3962)  loss_classifier: 0.1447 (0.1588)  loss_box_reg: 0.1547 (0.1557)  loss_objectness: 0.0251 (0.0284)  loss_rpn_box_reg: 0.0520 (0.0533)  time: 0.5371  data: 0.0144  max mem: 6482\n",
      "Epoch: [4]  [ 700/2699]  eta: 0:17:23  lr: 0.000500  loss: 0.4281 (0.3971)  loss_classifier: 0.1657 (0.1594)  loss_box_reg: 0.1617 (0.1557)  loss_objectness: 0.0291 (0.0287)  loss_rpn_box_reg: 0.0510 (0.0533)  time: 0.5262  data: 0.0115  max mem: 6482\n",
      "Epoch: [4]  [ 800/2699]  eta: 0:16:31  lr: 0.000500  loss: 0.3858 (0.3977)  loss_classifier: 0.1613 (0.1597)  loss_box_reg: 0.1511 (0.1558)  loss_objectness: 0.0205 (0.0288)  loss_rpn_box_reg: 0.0440 (0.0534)  time: 0.5194  data: 0.0116  max mem: 6482\n",
      "Epoch: [4]  [ 900/2699]  eta: 0:15:38  lr: 0.000500  loss: 0.4047 (0.3980)  loss_classifier: 0.1603 (0.1600)  loss_box_reg: 0.1530 (0.1557)  loss_objectness: 0.0256 (0.0288)  loss_rpn_box_reg: 0.0582 (0.0535)  time: 0.5152  data: 0.0115  max mem: 6482\n",
      "Epoch: [4]  [1000/2699]  eta: 0:14:45  lr: 0.000500  loss: 0.4295 (0.3991)  loss_classifier: 0.1718 (0.1606)  loss_box_reg: 0.1610 (0.1559)  loss_objectness: 0.0259 (0.0289)  loss_rpn_box_reg: 0.0483 (0.0537)  time: 0.5152  data: 0.0113  max mem: 6482\n",
      "Epoch: [4]  [1100/2699]  eta: 0:13:53  lr: 0.000500  loss: 0.4394 (0.3995)  loss_classifier: 0.1650 (0.1606)  loss_box_reg: 0.1633 (0.1561)  loss_objectness: 0.0199 (0.0290)  loss_rpn_box_reg: 0.0540 (0.0538)  time: 0.5186  data: 0.0120  max mem: 6482\n",
      "Epoch: [4]  [1200/2699]  eta: 0:13:01  lr: 0.000500  loss: 0.4075 (0.3984)  loss_classifier: 0.1664 (0.1603)  loss_box_reg: 0.1552 (0.1555)  loss_objectness: 0.0304 (0.0290)  loss_rpn_box_reg: 0.0481 (0.0536)  time: 0.5157  data: 0.0118  max mem: 6482\n",
      "Epoch: [4]  [1300/2699]  eta: 0:12:09  lr: 0.000500  loss: 0.3885 (0.3980)  loss_classifier: 0.1591 (0.1601)  loss_box_reg: 0.1605 (0.1553)  loss_objectness: 0.0246 (0.0290)  loss_rpn_box_reg: 0.0537 (0.0535)  time: 0.5270  data: 0.0151  max mem: 6482\n",
      "Epoch: [4]  [1400/2699]  eta: 0:11:16  lr: 0.000500  loss: 0.4113 (0.3979)  loss_classifier: 0.1678 (0.1600)  loss_box_reg: 0.1518 (0.1553)  loss_objectness: 0.0235 (0.0290)  loss_rpn_box_reg: 0.0565 (0.0536)  time: 0.5310  data: 0.0129  max mem: 6482\n",
      "Epoch: [4]  [1500/2699]  eta: 0:10:24  lr: 0.000500  loss: 0.4173 (0.3974)  loss_classifier: 0.1719 (0.1599)  loss_box_reg: 0.1521 (0.1552)  loss_objectness: 0.0296 (0.0289)  loss_rpn_box_reg: 0.0556 (0.0534)  time: 0.5161  data: 0.0115  max mem: 6482\n",
      "Epoch: [4]  [1600/2699]  eta: 0:09:32  lr: 0.000500  loss: 0.4285 (0.3966)  loss_classifier: 0.1658 (0.1596)  loss_box_reg: 0.1584 (0.1549)  loss_objectness: 0.0257 (0.0287)  loss_rpn_box_reg: 0.0476 (0.0534)  time: 0.5174  data: 0.0117  max mem: 6482\n",
      "Epoch: [4]  [1700/2699]  eta: 0:08:40  lr: 0.000500  loss: 0.4103 (0.3962)  loss_classifier: 0.1637 (0.1596)  loss_box_reg: 0.1457 (0.1546)  loss_objectness: 0.0214 (0.0287)  loss_rpn_box_reg: 0.0588 (0.0534)  time: 0.5159  data: 0.0113  max mem: 6482\n",
      "Epoch: [4]  [1800/2699]  eta: 0:07:48  lr: 0.000500  loss: 0.4007 (0.3953)  loss_classifier: 0.1566 (0.1593)  loss_box_reg: 0.1508 (0.1543)  loss_objectness: 0.0265 (0.0285)  loss_rpn_box_reg: 0.0500 (0.0532)  time: 0.5147  data: 0.0113  max mem: 6482\n",
      "Epoch: [4]  [1900/2699]  eta: 0:06:56  lr: 0.000500  loss: 0.3491 (0.3948)  loss_classifier: 0.1450 (0.1591)  loss_box_reg: 0.1403 (0.1540)  loss_objectness: 0.0250 (0.0285)  loss_rpn_box_reg: 0.0478 (0.0532)  time: 0.5175  data: 0.0120  max mem: 6482\n",
      "Epoch: [4]  [2000/2699]  eta: 0:06:04  lr: 0.000500  loss: 0.3849 (0.3946)  loss_classifier: 0.1582 (0.1590)  loss_box_reg: 0.1590 (0.1539)  loss_objectness: 0.0278 (0.0284)  loss_rpn_box_reg: 0.0394 (0.0531)  time: 0.5186  data: 0.0118  max mem: 6482\n",
      "Epoch: [4]  [2100/2699]  eta: 0:05:12  lr: 0.000500  loss: 0.4126 (0.3951)  loss_classifier: 0.1783 (0.1593)  loss_box_reg: 0.1710 (0.1541)  loss_objectness: 0.0330 (0.0285)  loss_rpn_box_reg: 0.0535 (0.0532)  time: 0.5383  data: 0.0141  max mem: 6482\n",
      "Epoch: [4]  [2200/2699]  eta: 0:04:21  lr: 0.000500  loss: 0.3247 (0.3942)  loss_classifier: 0.1414 (0.1590)  loss_box_reg: 0.1290 (0.1536)  loss_objectness: 0.0179 (0.0284)  loss_rpn_box_reg: 0.0459 (0.0531)  time: 0.7936  data: 0.2572  max mem: 6482\n",
      "Epoch: [4]  [2300/2699]  eta: 0:03:28  lr: 0.000500  loss: 0.3976 (0.3942)  loss_classifier: 0.1590 (0.1589)  loss_box_reg: 0.1518 (0.1537)  loss_objectness: 0.0263 (0.0284)  loss_rpn_box_reg: 0.0529 (0.0532)  time: 0.5156  data: 0.0116  max mem: 6482\n",
      "Epoch: [4]  [2400/2699]  eta: 0:02:36  lr: 0.000500  loss: 0.4062 (0.3937)  loss_classifier: 0.1518 (0.1588)  loss_box_reg: 0.1590 (0.1535)  loss_objectness: 0.0243 (0.0284)  loss_rpn_box_reg: 0.0484 (0.0530)  time: 0.5167  data: 0.0117  max mem: 6482\n",
      "Epoch: [4]  [2500/2699]  eta: 0:01:44  lr: 0.000500  loss: 0.3854 (0.3938)  loss_classifier: 0.1595 (0.1589)  loss_box_reg: 0.1506 (0.1534)  loss_objectness: 0.0233 (0.0284)  loss_rpn_box_reg: 0.0564 (0.0531)  time: 0.5158  data: 0.0110  max mem: 6482\n",
      "Epoch: [4]  [2600/2699]  eta: 0:00:51  lr: 0.000500  loss: 0.3813 (0.3932)  loss_classifier: 0.1602 (0.1587)  loss_box_reg: 0.1519 (0.1532)  loss_objectness: 0.0200 (0.0283)  loss_rpn_box_reg: 0.0481 (0.0530)  time: 0.5146  data: 0.0116  max mem: 6482\n",
      "Epoch: [4]  [2698/2699]  eta: 0:00:00  lr: 0.000500  loss: 0.3336 (0.3928)  loss_classifier: 0.1343 (0.1585)  loss_box_reg: 0.1418 (0.1533)  loss_objectness: 0.0198 (0.0282)  loss_rpn_box_reg: 0.0424 (0.0528)  time: 0.4998  data: 0.0112  max mem: 6482\n",
      "Epoch: [4] Total time: 0:23:31 (0.5231 s / it)\n",
      "creating index...\n",
      "index created!\n",
      "Test:  [  0/675]  eta: 0:17:41  model_time: 0.2494 (0.2494)  evaluator_time: 0.3875 (0.3875)  time: 1.5725  data: 0.8875  max mem: 6482\n",
      "Test:  [100/675]  eta: 0:05:17  model_time: 0.1516 (0.1538)  evaluator_time: 0.2938 (0.3680)  time: 0.5007  data: 0.0136  max mem: 6482\n",
      "Test:  [200/675]  eta: 0:05:05  model_time: 0.1519 (0.1533)  evaluator_time: 0.7796 (0.4638)  time: 0.9931  data: 0.0120  max mem: 6482\n",
      "Test:  [300/675]  eta: 0:03:52  model_time: 0.1510 (0.1540)  evaluator_time: 0.4183 (0.4425)  time: 0.6171  data: 0.0129  max mem: 6482\n",
      "Test:  [400/675]  eta: 0:02:43  model_time: 0.1520 (0.1544)  evaluator_time: 0.4501 (0.4149)  time: 0.6809  data: 0.0123  max mem: 6482\n",
      "Test:  [500/675]  eta: 0:01:43  model_time: 0.1503 (0.1541)  evaluator_time: 0.9554 (0.4160)  time: 1.0876  data: 0.0124  max mem: 6482\n",
      "Test:  [600/675]  eta: 0:00:45  model_time: 0.1587 (0.1546)  evaluator_time: 0.1515 (0.4304)  time: 0.4571  data: 0.0121  max mem: 6482\n",
      "Test:  [674/675]  eta: 0:00:00  model_time: 0.1597 (0.1547)  evaluator_time: 0.0881 (0.4132)  time: 0.2676  data: 0.0110  max mem: 6482\n",
      "Test: Total time: 0:06:38 (0.5909 s / it)\n",
      "Averaged stats: model_time: 0.1597 (0.1547)  evaluator_time: 0.0881 (0.4132)\n",
      "Accumulating evaluation results...\n",
      "DONE (t=1.73s).\n",
      "IoU metric: bbox\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.468\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.899\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.430\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.048\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.458\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.532\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.016\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.149\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.536\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.140\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.525\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.600\n"
     ]
    }
   ],
   "source": [
    "num_classes = 2\n",
    "train_dataset = WheatDataset(train_df, folds=[0, 2, 3, 4])\n",
    "train_loader = data_utils.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4, collate_fn=utils.collate_fn)\n",
    "\n",
    "val_dataset = WheatDataset(train_df, folds=[1])\n",
    "val_loader = data_utils.DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4, collate_fn=utils.collate_fn)\n",
    "\n",
    "\n",
    "# move model to the right device\n",
    "model.to(DEVICE)\n",
    "\n",
    "# construct an optimizer\n",
    "params = [p for p in model.parameters() if p.requires_grad]\n",
    "optimizer = torch.optim.SGD(params, lr=0.005, momentum=0.9, weight_decay=0.0005)\n",
    "\n",
    "# and a learning rate scheduler\n",
    "lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1)\n",
    "\n",
    "for epoch in range(N_EPOCHS):\n",
    "    # train for one epoch, printing every 100 iterations\n",
    "    train_one_epoch(model, optimizer, train_loader, DEVICE, epoch, print_freq=100)\n",
    "    # update the learning rate\n",
    "    lr_scheduler.step()\n",
    "    # evaluate on the validation dataset\n",
    "    evaluate(model, val_loader, device=DEVICE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), 'fasterrcnn_resnet101_fold1.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_bbox(bboxes, col, color='white'):\n",
    "    \n",
    "    for i in range(len(bboxes)):\n",
    "        # Create a Rectangle patch\n",
    "        rect = patches.Rectangle(\n",
    "            (bboxes[i][0], bboxes[i][1]),\n",
    "            bboxes[i][2] - bboxes[i][0], \n",
    "            bboxes[i][3] - bboxes[i][1], \n",
    "            linewidth=2, \n",
    "            edgecolor=color, \n",
    "            facecolor='none')\n",
    "\n",
    "        # Add the patch to the Axes\n",
    "        col.add_patch(rect)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'device' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-14-e23ba2552ea7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0mimage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcv2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcvtColor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcv2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCOLOR_BGR2RGB\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mimage\u001b[0m \u001b[0;34m/=\u001b[0m \u001b[0;36m255.0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m     \u001b[0mpreds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0mpred_bboxes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpreds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'boxes'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdetach\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'device' is not defined"
     ]
    }
   ],
   "source": [
    "for img in os.listdir(TEST_DIR)[:5]:\n",
    "    image = cv2.imread(os.path.join(TEST_DIR, img), cv2.IMREAD_COLOR)\n",
    "    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n",
    "    image /= 255.0\n",
    "    preds = model([torch.from_numpy(np.transpose(image, (2, 0, 1))).to(device)])[0]\n",
    "    \n",
    "    pred_bboxes = preds['boxes'].cpu().detach().numpy()\n",
    "    pred_scores = preds['scores'].cpu().detach().numpy()\n",
    "    \n",
    "    mask = pred_scores >= 0.4\n",
    "    pred_scores = pred_scores[mask]\n",
    "    pred_bboxes = pred_bboxes[mask]\n",
    "    \n",
    "    fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(10, 10))\n",
    "    get_bbox(pred_bboxes, ax, color='red')\n",
    "    ax.imshow(image)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "50bf8420e0934d59aae87b67523a15e7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "7662b01928ac45178181ec4795c32467": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_ee0ce4d208364219bdb61d92dbae9e64",
       "max": 178728960.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_e71dcdbe6141480f808b0b810567ad0c",
       "value": 178728960.0
      }
     },
     "ba7a0dd8438d4188a31783056ce95667": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "d6d8a168f54f49f3903362a57794fd44": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_7662b01928ac45178181ec4795c32467",
        "IPY_MODEL_db10c0310ead47f39f9b270b46f42b58"
       ],
       "layout": "IPY_MODEL_da0cf36bb7474024904594b50ee142ae"
      }
     },
     "da0cf36bb7474024904594b50ee142ae": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "db10c0310ead47f39f9b270b46f42b58": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_50bf8420e0934d59aae87b67523a15e7",
       "placeholder": "​",
       "style": "IPY_MODEL_ba7a0dd8438d4188a31783056ce95667",
       "value": " 170M/170M [00:25&lt;00:00, 6.94MB/s]"
      }
     },
     "e71dcdbe6141480f808b0b810567ad0c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "ee0ce4d208364219bdb61d92dbae9e64": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
